In [ ]:
"""
Import Module
"""
import pandas as pd
import plotly
from datetime import datetime
from tensorflow.keras.datasets import cifar10
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import EarlyStopping
import numpy as np
from sklearn.metrics import mean_squared_error,mean_absolute_error
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt
from tensorflow import keras
from tensorflow.keras import Input, datasets, layers, models
import tensorflow as tf
from tensorflow.keras.layers import ReLU,add, LeakyReLU
from keras.layers import Conv2D, Conv2DTranspose, UpSampling2D, MaxPool2D, Flatten, BatchNormalization
from keras.layers import Dropout, Add, Concatenate
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import SGD, Adam, RMSprop, Adadelta
from keras.layers import MaxPooling2D, UpSampling2D, BatchNormalization, Activation
from tensorflow.keras import regularizers
from tensorflow.keras.callbacks import ModelCheckpoint, Callback, LearningRateScheduler
from tensorflow.keras import layers
import os
import random

De-Noising Model¶

Autoencoder with CNN¶

In [ ]:
"""
Collecting all the functions under a code block
"""
def preprocess_image(array):
    array = array.astype("float32") / 255.0
    array = np.reshape(array, (len(array), 32, 32, 3))
    return array


def adding_noise(array):
    noise_factor = 0.3
    noisy_array = array + noise_factor * np.random.normal(
        loc=0.0, scale=1, size=array.shape
    )
    return np.clip(noisy_array, 0.0, 1.0)


def display_images(array1, array2):
    n = 10

    indices = np.random.randint(len(array1), size=n)
    images1 = array1[indices, :]
    images2 = array2[indices, :]

    plt.figure(figsize=(20, 4))
    for i, (image1, image2) in enumerate(zip(images1, images2)):
        ax = plt.subplot(2, n, i + 1)
        plt.imshow(image1)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

        ax = plt.subplot(2, n, i + 1 + n)
        plt.imshow(image2)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

    plt.show()
In [ ]:
"""
Download the dataset
"""
img_rows, img_cols, img_ch = 32, 32, 3
(x_train, train_labels), (x_test, test_labels) = datasets.cifar10.load_data()
Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
170500096/170498071 [==============================] - 3s 0us/step
170508288/170498071 [==============================] - 3s 0us/step
In [ ]:
"""
Print shape and sample images from the dataset
"""
print('Train: X=%s, y=%s' % (x_train.shape, train_labels.shape))
print('Test: X=%s, y=%s' % (x_test.shape, test_labels.shape))

plt.figure(figsize=(10,10))
for index in range(50):
    plt.subplot(10,10,index+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(x_train[index])
plt.show()
Train: X=(50000, 32, 32, 3), y=(50000, 1)
Test: X=(10000, 32, 32, 3), y=(10000, 1)
In [ ]:
train_data = preprocess_image(x_train)
test_data = preprocess_image(x_test)

# Create a copy of the data with added noise
noisy_train_data = adding_noise(train_data)
noisy_test_data = adding_noise(test_data)

# Display the train data and a version of it with added noise
print(f"Training dataset size = {train_data.shape}\nTesting dataset size = {test_data.shape}")
print(f"Noisy Training dataset = {noisy_train_data.shape}\nNosisy Testing dataset = {noisy_test_data.shape}")
print(f"\n\nRegular vs Noisy Images -")
print(f"{display_images(x_train, noisy_train_data)}")
Training dataset size = (50000, 32, 32, 3)
Testing dataset size = (10000, 32, 32, 3)
Noisy Training dataset = (50000, 32, 32, 3)
Nosisy Testing dataset = (10000, 32, 32, 3)


Regular vs Noisy Images -
None

Model 1¶

In [ ]:
input_img = Input((32,32,3))

# Encoder
encoder = Conv2D(32, 3,activation='relu', padding="same")(input_img)
encoder = BatchNormalization()(encoder)
encoder= MaxPool2D(2)(encoder)
encoder = Conv2D(64, 3,activation='relu', padding="same")(encoder)
encoder = BatchNormalization()(encoder)
encoder = MaxPool2D(2)(encoder)

#Middle
mid = Conv2D(128, 3,activation='relu', padding="same")(encoder)
mid = BatchNormalization()(mid)

# Decoder
up1 = UpSampling2D((2,2))(mid)
decoder = Conv2D(64, 3,activation='relu', padding="same")(up1)
decoder = BatchNormalization()(decoder)
up2 = UpSampling2D((2,2))(decoder)
decoder = Conv2D(32, 3,activation='relu', padding="same")(up2)
decoder= BatchNormalization()(decoder)

# output
decoder = Conv2D(3, 1)(decoder)
output = Activation("sigmoid")(decoder)

model1 = Model(input_img, output)
model1.summary()
Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_1 (InputLayer)        [(None, 32, 32, 3)]       0         
                                                                 
 conv2d (Conv2D)             (None, 32, 32, 32)        896       
                                                                 
 batch_normalization (BatchN  (None, 32, 32, 32)       128       
 ormalization)                                                   
                                                                 
 max_pooling2d (MaxPooling2D  (None, 16, 16, 32)       0         
 )                                                               
                                                                 
 conv2d_1 (Conv2D)           (None, 16, 16, 64)        18496     
                                                                 
 batch_normalization_1 (Batc  (None, 16, 16, 64)       256       
 hNormalization)                                                 
                                                                 
 max_pooling2d_1 (MaxPooling  (None, 8, 8, 64)         0         
 2D)                                                             
                                                                 
 conv2d_2 (Conv2D)           (None, 8, 8, 128)         73856     
                                                                 
 batch_normalization_2 (Batc  (None, 8, 8, 128)        512       
 hNormalization)                                                 
                                                                 
 up_sampling2d (UpSampling2D  (None, 16, 16, 128)      0         
 )                                                               
                                                                 
 conv2d_3 (Conv2D)           (None, 16, 16, 64)        73792     
                                                                 
 batch_normalization_3 (Batc  (None, 16, 16, 64)       256       
 hNormalization)                                                 
                                                                 
 up_sampling2d_1 (UpSampling  (None, 32, 32, 64)       0         
 2D)                                                             
                                                                 
 conv2d_4 (Conv2D)           (None, 32, 32, 32)        18464     
                                                                 
 batch_normalization_4 (Batc  (None, 32, 32, 32)       128       
 hNormalization)                                                 
                                                                 
 conv2d_5 (Conv2D)           (None, 32, 32, 3)         99        
                                                                 
 activation (Activation)     (None, 32, 32, 3)         0         
                                                                 
=================================================================
Total params: 186,883
Trainable params: 186,243
Non-trainable params: 640
_________________________________________________________________
In [ ]:
"""
Compile Model 1
"""
model1.compile(Adam(learning_rate=0.0001), metrics=["mae","accuracy"],loss='binary_crossentropy')
es_cb = EarlyStopping(
    monitor='val_loss', patience=2
)
history = model1.fit(
    noisy_train_data, train_data,
    batch_size=20,
    epochs=500,
    verbose=1,
    validation_data=(
        noisy_test_data, test_data
    ),
    callbacks=[es_cb],
    shuffle=True
)
Epoch 1/500
2500/2500 [==============================] - 57s 18ms/step - loss: 0.5783 - mae: 0.0818 - accuracy: 0.6339 - val_loss: 0.5685 - val_mae: 0.0662 - val_accuracy: 0.6917
Epoch 2/500
2500/2500 [==============================] - 42s 17ms/step - loss: 0.5698 - mae: 0.0704 - accuracy: 0.6783 - val_loss: 0.5664 - val_mae: 0.0626 - val_accuracy: 0.7066
Epoch 3/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5683 - mae: 0.0680 - accuracy: 0.6872 - val_loss: 0.5654 - val_mae: 0.0607 - val_accuracy: 0.7095
Epoch 4/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5676 - mae: 0.0669 - accuracy: 0.6902 - val_loss: 0.5648 - val_mae: 0.0599 - val_accuracy: 0.7116
Epoch 5/500
2500/2500 [==============================] - 41s 17ms/step - loss: 0.5669 - mae: 0.0656 - accuracy: 0.6929 - val_loss: 0.5647 - val_mae: 0.0594 - val_accuracy: 0.7173
Epoch 6/500
2500/2500 [==============================] - 42s 17ms/step - loss: 0.5666 - mae: 0.0652 - accuracy: 0.6939 - val_loss: 0.5644 - val_mae: 0.0592 - val_accuracy: 0.7070
Epoch 7/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5663 - mae: 0.0647 - accuracy: 0.6952 - val_loss: 0.5641 - val_mae: 0.0586 - val_accuracy: 0.7190
Epoch 8/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5661 - mae: 0.0643 - accuracy: 0.6958 - val_loss: 0.5640 - val_mae: 0.0585 - val_accuracy: 0.7200
Epoch 9/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5659 - mae: 0.0640 - accuracy: 0.6974 - val_loss: 0.5639 - val_mae: 0.0583 - val_accuracy: 0.7212
Epoch 10/500
2500/2500 [==============================] - 47s 19ms/step - loss: 0.5658 - mae: 0.0638 - accuracy: 0.6981 - val_loss: 0.5639 - val_mae: 0.0583 - val_accuracy: 0.7217
Epoch 11/500
2500/2500 [==============================] - 43s 17ms/step - loss: 0.5657 - mae: 0.0635 - accuracy: 0.6998 - val_loss: 0.5638 - val_mae: 0.0580 - val_accuracy: 0.7243
Epoch 12/500
2500/2500 [==============================] - 41s 16ms/step - loss: 0.5655 - mae: 0.0632 - accuracy: 0.7006 - val_loss: 0.5637 - val_mae: 0.0577 - val_accuracy: 0.7215
Epoch 13/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5654 - mae: 0.0630 - accuracy: 0.7009 - val_loss: 0.5638 - val_mae: 0.0581 - val_accuracy: 0.7167
Epoch 14/500
2500/2500 [==============================] - 46s 18ms/step - loss: 0.5653 - mae: 0.0629 - accuracy: 0.7006 - val_loss: 0.5636 - val_mae: 0.0577 - val_accuracy: 0.7203
Epoch 15/500
2500/2500 [==============================] - 47s 19ms/step - loss: 0.5653 - mae: 0.0627 - accuracy: 0.7009 - val_loss: 0.5635 - val_mae: 0.0575 - val_accuracy: 0.7159
Epoch 16/500
2500/2500 [==============================] - 39s 15ms/step - loss: 0.5651 - mae: 0.0624 - accuracy: 0.7018 - val_loss: 0.5635 - val_mae: 0.0572 - val_accuracy: 0.7198
Epoch 17/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5651 - mae: 0.0623 - accuracy: 0.7022 - val_loss: 0.5634 - val_mae: 0.0573 - val_accuracy: 0.7238
Epoch 18/500
2500/2500 [==============================] - 39s 15ms/step - loss: 0.5651 - mae: 0.0623 - accuracy: 0.7032 - val_loss: 0.5634 - val_mae: 0.0572 - val_accuracy: 0.7185
Epoch 19/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5651 - mae: 0.0623 - accuracy: 0.7035 - val_loss: 0.5634 - val_mae: 0.0571 - val_accuracy: 0.7229
Epoch 20/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5650 - mae: 0.0621 - accuracy: 0.7041 - val_loss: 0.5633 - val_mae: 0.0570 - val_accuracy: 0.7240
Epoch 21/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5650 - mae: 0.0621 - accuracy: 0.7040 - val_loss: 0.5634 - val_mae: 0.0574 - val_accuracy: 0.7247
Epoch 22/500
2500/2500 [==============================] - 45s 18ms/step - loss: 0.5650 - mae: 0.0621 - accuracy: 0.7043 - val_loss: 0.5634 - val_mae: 0.0570 - val_accuracy: 0.7209
In [ ]:
"""
Model 1 evaluation
"""
score = model1.evaluate(noisy_test_data, test_data)
print(score)
prediction_data = model1.predict(noisy_test_data)
print(f"Noisy test data and predicted output")
print(f"{display_images(noisy_test_data, prediction_data)}")
313/313 [==============================] - 4s 10ms/step - loss: 0.5634 - mae: 0.0570 - accuracy: 0.7209
[0.5633975863456726, 0.05696729198098183, 0.7208859324455261]
Noisy test data and predicted output
None
In [ ]:
index = [
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data))
]

for i in index:
    plt.subplot(1,3,1)
    plt.imshow(test_data[i])
    plt.title('original')
    plt.subplot(1,3,2)
    plt.imshow(noisy_test_data[i])
    plt.title('noisy')
    plt.subplot(1,3,3)
    plt.imshow(prediction_data[i])
    plt.title('denoised')
    plt.show()
In [ ]:
f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history.epoch, history.history['loss'], label = "loss") 
plt.plot(history.epoch, history.history['val_loss'], label = "val_loss")

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("Loss",fontsize=10)
plt.legend()
plt.show()

f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history.epoch, history.history['mae'], label = "mae") 
plt.plot(history.epoch, history.history['val_mae'], label = "val_mae") 

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("MAE",fontsize=10)
plt.legend()
plt.show()

Model 2¶

In [ ]:
input_img = Input(shape=(32, 32, 3))

# Encoder
x = Conv2D(32, (3, 3), activation="relu", padding="same")(input_img)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)
x = Dropout(0.5)(x)
x = Conv2D(64, (3, 3), activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)
x = Dropout(0.5)(x)
x = Conv2D(128, (3, 3), activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)

# Decoder
x = Conv2DTranspose(128, (3, 3), strides=2, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
x = Conv2DTranspose(64, (3, 3), strides=2, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)

x = Conv2DTranspose(32, (3, 3), strides=2, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)

x = Conv2D(3, (3, 3), activation="sigmoid", padding="same")(x)

model2 = Model(input_img, x)
model2.summary()
Model: "model_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_2 (InputLayer)        [(None, 32, 32, 3)]       0         
                                                                 
 conv2d_6 (Conv2D)           (None, 32, 32, 32)        896       
                                                                 
 batch_normalization_5 (Batc  (None, 32, 32, 32)       128       
 hNormalization)                                                 
                                                                 
 max_pooling2d_2 (MaxPooling  (None, 16, 16, 32)       0         
 2D)                                                             
                                                                 
 dropout (Dropout)           (None, 16, 16, 32)        0         
                                                                 
 conv2d_7 (Conv2D)           (None, 16, 16, 64)        18496     
                                                                 
 batch_normalization_6 (Batc  (None, 16, 16, 64)       256       
 hNormalization)                                                 
                                                                 
 max_pooling2d_3 (MaxPooling  (None, 8, 8, 64)         0         
 2D)                                                             
                                                                 
 dropout_1 (Dropout)         (None, 8, 8, 64)          0         
                                                                 
 conv2d_8 (Conv2D)           (None, 8, 8, 128)         73856     
                                                                 
 batch_normalization_7 (Batc  (None, 8, 8, 128)        512       
 hNormalization)                                                 
                                                                 
 max_pooling2d_4 (MaxPooling  (None, 4, 4, 128)        0         
 2D)                                                             
                                                                 
 conv2d_transpose (Conv2DTra  (None, 8, 8, 128)        147584    
 nspose)                                                         
                                                                 
 batch_normalization_8 (Batc  (None, 8, 8, 128)        512       
 hNormalization)                                                 
                                                                 
 dropout_2 (Dropout)         (None, 8, 8, 128)         0         
                                                                 
 conv2d_transpose_1 (Conv2DT  (None, 16, 16, 64)       73792     
 ranspose)                                                       
                                                                 
 batch_normalization_9 (Batc  (None, 16, 16, 64)       256       
 hNormalization)                                                 
                                                                 
 dropout_3 (Dropout)         (None, 16, 16, 64)        0         
                                                                 
 conv2d_transpose_2 (Conv2DT  (None, 32, 32, 32)       18464     
 ranspose)                                                       
                                                                 
 batch_normalization_10 (Bat  (None, 32, 32, 32)       128       
 chNormalization)                                                
                                                                 
 leaky_re_lu (LeakyReLU)     (None, 32, 32, 32)        0         
                                                                 
 conv2d_9 (Conv2D)           (None, 32, 32, 3)         867       
                                                                 
=================================================================
Total params: 335,747
Trainable params: 334,851
Non-trainable params: 896
_________________________________________________________________
In [ ]:
epochs = 20
batch_size = 500
es = EarlyStopping(monitor='val_loss', patience=2)

model2.compile(optimizer=Adam(learning_rate=0.0001),metrics=['mae', 'accuracy'], loss='binary_crossentropy')

history2 = model2.fit(
    noisy_train_data, train_data,
    batch_size=20,
    epochs=500, 
    verbose=1,
    validation_data=(
        noisy_test_data, 
        test_data
      ),
    callbacks=[es],
    shuffle=True
    )
Epoch 1/500
2500/2500 [==============================] - 50s 19ms/step - loss: 0.6114 - mae: 0.1196 - accuracy: 0.5079 - val_loss: 0.5906 - val_mae: 0.1005 - val_accuracy: 0.6091
Epoch 2/500
2500/2500 [==============================] - ETA: 0s - loss: 0.5887 - mae: 0.0968 - accuracy: 0.5852Epoch 3/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5850 - mae: 0.0921 - accuracy: 0.5954 - val_loss: 0.5786 - val_mae: 0.0823 - val_accuracy: 0.6254
Epoch 4/500
2500/2500 [==============================] - 36s 14ms/step - loss: 0.5831 - mae: 0.0896 - accuracy: 0.5994 - val_loss: 0.5787 - val_mae: 0.0826 - val_accuracy: 0.6268
Epoch 5/500
2500/2500 [==============================] - 36s 14ms/step - loss: 0.5819 - mae: 0.0880 - accuracy: 0.6019 - val_loss: 0.5762 - val_mae: 0.0787 - val_accuracy: 0.6285
Epoch 6/500
2500/2500 [==============================] - 36s 15ms/step - loss: 0.5811 - mae: 0.0869 - accuracy: 0.6091 - val_loss: 0.5752 - val_mae: 0.0768 - val_accuracy: 0.6399
Epoch 7/500
2500/2500 [==============================] - 37s 15ms/step - loss: 0.5803 - mae: 0.0857 - accuracy: 0.6247 - val_loss: 0.5738 - val_mae: 0.0744 - val_accuracy: 0.6572
Epoch 8/500
2500/2500 [==============================] - 37s 15ms/step - loss: 0.5796 - mae: 0.0848 - accuracy: 0.6334 - val_loss: 0.5734 - val_mae: 0.0737 - val_accuracy: 0.6739
Epoch 9/500
2500/2500 [==============================] - 37s 15ms/step - loss: 0.5791 - mae: 0.0841 - accuracy: 0.6372 - val_loss: 0.5731 - val_mae: 0.0733 - val_accuracy: 0.6853
Epoch 10/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5788 - mae: 0.0836 - accuracy: 0.6400 - val_loss: 0.5720 - val_mae: 0.0711 - val_accuracy: 0.6837
Epoch 11/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5783 - mae: 0.0828 - accuracy: 0.6423 - val_loss: 0.5720 - val_mae: 0.0713 - val_accuracy: 0.6832
Epoch 12/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5781 - mae: 0.0826 - accuracy: 0.6427 - val_loss: 0.5727 - val_mae: 0.0726 - val_accuracy: 0.6934
In [ ]:
score = model2.evaluate(noisy_test_data, test_data)
print(score)
prediction_data = model2.predict(noisy_test_data)
print(f"Noisy test data and predicted output")
print(f"{display_images(noisy_test_data, prediction_data)}")
313/313 [==============================] - 4s 12ms/step - loss: 0.5727 - mae: 0.0726 - accuracy: 0.6934
[0.5726848244667053, 0.07260323315858841, 0.6933776140213013]
Noisy test data and predicted output
None
In [ ]:
index = [
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data))
]

for i in index:
    plt.subplot(1,3,1)
    plt.imshow(test_data[i])
    plt.title('original')
    plt.subplot(1,3,2)
    plt.imshow(noisy_test_data[i])
    plt.title('noisy')
    plt.subplot(1,3,3)
    plt.imshow(prediction_data[i])
    plt.title('denoised')
    plt.show()
In [ ]:
f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history2.epoch, history2.history['loss'], label = "loss") 
plt.plot(history2.epoch, history2.history['val_loss'], label = "val_loss")

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("Loss",fontsize=10)
plt.legend()
plt.show()

f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history2.epoch, history2.history['mae'], label = "mae") 
plt.plot(history2.epoch, history2.history['val_mae'], label = "val_mae") 

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("MAE",fontsize=10)
plt.legend()
plt.show()

Model 3¶

In [ ]:
# Encoder
input_img = Input(shape=(32, 32, 3))

x = Conv2D(32, (3, 3), activation="relu", padding="same")(input_img)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)
x = Dropout(0.3)(x)
x = Conv2D(64, (3, 3), activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)
x = Dropout(0.3)(x)
x = Conv2D(128, (3, 3), activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2))(x)

# Decoder
x = Conv2DTranspose(128, (3, 3), strides=2, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = Dropout(0.3)(x)
x = Conv2DTranspose(64, (3, 3), strides=2, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = Dropout(0.3)(x)

x = Conv2DTranspose(32, (3, 3), strides=2, activation="relu", padding="same")(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)

x = Conv2D(3, (3, 3), activation="sigmoid", padding="same")(x)

decoded = Conv2DTranspose(3, 3, activation='sigmoid', padding='same')(x)

model3 = Model(input_img, decoded)
model3.summary()
Model: "model_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_3 (InputLayer)        [(None, 32, 32, 3)]       0         
                                                                 
 conv2d_10 (Conv2D)          (None, 32, 32, 32)        896       
                                                                 
 batch_normalization_11 (Bat  (None, 32, 32, 32)       128       
 chNormalization)                                                
                                                                 
 max_pooling2d_5 (MaxPooling  (None, 16, 16, 32)       0         
 2D)                                                             
                                                                 
 dropout_4 (Dropout)         (None, 16, 16, 32)        0         
                                                                 
 conv2d_11 (Conv2D)          (None, 16, 16, 64)        18496     
                                                                 
 batch_normalization_12 (Bat  (None, 16, 16, 64)       256       
 chNormalization)                                                
                                                                 
 max_pooling2d_6 (MaxPooling  (None, 8, 8, 64)         0         
 2D)                                                             
                                                                 
 dropout_5 (Dropout)         (None, 8, 8, 64)          0         
                                                                 
 conv2d_12 (Conv2D)          (None, 8, 8, 128)         73856     
                                                                 
 batch_normalization_13 (Bat  (None, 8, 8, 128)        512       
 chNormalization)                                                
                                                                 
 max_pooling2d_7 (MaxPooling  (None, 4, 4, 128)        0         
 2D)                                                             
                                                                 
 conv2d_transpose_3 (Conv2DT  (None, 8, 8, 128)        147584    
 ranspose)                                                       
                                                                 
 batch_normalization_14 (Bat  (None, 8, 8, 128)        512       
 chNormalization)                                                
                                                                 
 dropout_6 (Dropout)         (None, 8, 8, 128)         0         
                                                                 
 conv2d_transpose_4 (Conv2DT  (None, 16, 16, 64)       73792     
 ranspose)                                                       
                                                                 
 batch_normalization_15 (Bat  (None, 16, 16, 64)       256       
 chNormalization)                                                
                                                                 
 dropout_7 (Dropout)         (None, 16, 16, 64)        0         
                                                                 
 conv2d_transpose_5 (Conv2DT  (None, 32, 32, 32)       18464     
 ranspose)                                                       
                                                                 
 batch_normalization_16 (Bat  (None, 32, 32, 32)       128       
 chNormalization)                                                
                                                                 
 leaky_re_lu_1 (LeakyReLU)   (None, 32, 32, 32)        0         
                                                                 
 conv2d_13 (Conv2D)          (None, 32, 32, 3)         867       
                                                                 
 conv2d_transpose_6 (Conv2DT  (None, 32, 32, 3)        84        
 ranspose)                                                       
                                                                 
=================================================================
Total params: 335,831
Trainable params: 334,935
Non-trainable params: 896
_________________________________________________________________
In [ ]:
model3.compile(optimizer=Adam(learning_rate=0.0001),metrics=['mae','accuracy'], loss="binary_crossentropy")

es = EarlyStopping(monitor='val_loss', patience=2)
history3 = model3.fit(
    noisy_train_data, 
    train_data,
    batch_size=20,
    epochs=500,
    verbose=1,
    validation_data=(
        noisy_test_data, 
        test_data
        ),
    callbacks=[es],
    shuffle=True
  )
Epoch 1/500
2500/2500 [==============================] - 41s 16ms/step - loss: 0.6122 - mae: 0.1248 - accuracy: 0.5066 - val_loss: 0.5867 - val_mae: 0.0934 - val_accuracy: 0.5774
Epoch 2/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5851 - mae: 0.0924 - accuracy: 0.5759 - val_loss: 0.5787 - val_mae: 0.0820 - val_accuracy: 0.6093
Epoch 3/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5810 - mae: 0.0869 - accuracy: 0.5928 - val_loss: 0.5756 - val_mae: 0.0773 - val_accuracy: 0.6193
Epoch 4/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5790 - mae: 0.0840 - accuracy: 0.5999 - val_loss: 0.5745 - val_mae: 0.0758 - val_accuracy: 0.6195
Epoch 5/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5777 - mae: 0.0821 - accuracy: 0.6042 - val_loss: 0.5738 - val_mae: 0.0749 - val_accuracy: 0.6259
Epoch 6/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5766 - mae: 0.0805 - accuracy: 0.6074 - val_loss: 0.5732 - val_mae: 0.0739 - val_accuracy: 0.6318
Epoch 7/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5758 - mae: 0.0793 - accuracy: 0.6108 - val_loss: 0.5724 - val_mae: 0.0727 - val_accuracy: 0.6334
Epoch 8/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5752 - mae: 0.0784 - accuracy: 0.6141 - val_loss: 0.5722 - val_mae: 0.0724 - val_accuracy: 0.6391
Epoch 9/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5748 - mae: 0.0777 - accuracy: 0.6157 - val_loss: 0.5717 - val_mae: 0.0716 - val_accuracy: 0.6386
Epoch 10/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5744 - mae: 0.0771 - accuracy: 0.6180 - val_loss: 0.5714 - val_mae: 0.0710 - val_accuracy: 0.6422
Epoch 11/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5741 - mae: 0.0766 - accuracy: 0.6186 - val_loss: 0.5713 - val_mae: 0.0709 - val_accuracy: 0.6421
Epoch 12/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5738 - mae: 0.0762 - accuracy: 0.6199 - val_loss: 0.5710 - val_mae: 0.0705 - val_accuracy: 0.6469
Epoch 13/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5735 - mae: 0.0758 - accuracy: 0.6208 - val_loss: 0.5708 - val_mae: 0.0702 - val_accuracy: 0.6442
Epoch 14/500
2500/2500 [==============================] - 39s 15ms/step - loss: 0.5732 - mae: 0.0754 - accuracy: 0.6215 - val_loss: 0.5703 - val_mae: 0.0693 - val_accuracy: 0.6471
Epoch 15/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5730 - mae: 0.0750 - accuracy: 0.6228 - val_loss: 0.5701 - val_mae: 0.0689 - val_accuracy: 0.6465
Epoch 16/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5729 - mae: 0.0748 - accuracy: 0.6227 - val_loss: 0.5701 - val_mae: 0.0688 - val_accuracy: 0.6486
Epoch 17/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5727 - mae: 0.0745 - accuracy: 0.6235 - val_loss: 0.5697 - val_mae: 0.0683 - val_accuracy: 0.6499
Epoch 18/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5725 - mae: 0.0742 - accuracy: 0.6247 - val_loss: 0.5697 - val_mae: 0.0683 - val_accuracy: 0.6496
Epoch 19/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5724 - mae: 0.0740 - accuracy: 0.6262 - val_loss: 0.5698 - val_mae: 0.0686 - val_accuracy: 0.6474
Epoch 20/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5722 - mae: 0.0738 - accuracy: 0.6286 - val_loss: 0.5697 - val_mae: 0.0683 - val_accuracy: 0.6549
In [ ]:
score = model3.evaluate(noisy_test_data, test_data)
print(score)
prediction_data = model3.predict(noisy_test_data)
print(f"Noisy test data and predicted output")
print(f"{display_images(noisy_test_data, prediction_data)}")
313/313 [==============================] - 4s 14ms/step - loss: 0.5697 - mae: 0.0683 - accuracy: 0.6549
[0.5697406530380249, 0.06833998113870621, 0.654900312423706]
Noisy test data and predicted output
None
In [ ]:
index = [
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data))
]

for i in index:
    plt.subplot(1,3,1)
    plt.imshow(test_data[i])
    plt.title('original')
    plt.subplot(1,3,2)
    plt.imshow(noisy_test_data[i])
    plt.title('noisy')
    plt.subplot(1,3,3)
    plt.imshow(prediction_data[i])
    plt.title('denoised')
    plt.show()
In [ ]:
f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history3.epoch, history3.history['loss'], label = "loss") 
plt.plot(history3.epoch, history3.history['val_loss'], label = "val_loss")

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("Loss",fontsize=10)
plt.legend()
plt.show()

f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history3.epoch, history3.history['mae'], label = "mae") 
plt.plot(history3.epoch, history3.history['val_mae'], label = "val_mae") 

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("MAE",fontsize=10)
plt.legend()
plt.show()

Model 4¶

In [ ]:
# Encoder
y = Conv2D(32, 3, activation='relu', padding='same')(input_img)
y = BatchNormalization()(y) #standarized the inputs
y = MaxPool2D()(y)
y = Dropout(0.5)(y)

y = Conv2D(64, 3, activation='relu', padding='same')(y)
y = BatchNormalization()(y)

skip_connection = Conv2D(64, 3, padding='same')(y) 
y = LeakyReLU()(skip_connection)  
y = BatchNormalization()(y)
y = MaxPool2D()(y)
y = Dropout(0.5)(y)

y = Conv2D(128, 3, activation='relu', padding='same')(y)
y = BatchNormalization()(y)
encoded = MaxPool2D()(y)

# Decoder
y = Conv2DTranspose(128, 3,activation='relu',strides=(2,2), padding='same')(encoded)
y = BatchNormalization()(y)
y = Conv2DTranspose(64, 3,activation='relu',strides=(2,2), padding='same')(y)
y = BatchNormalization()(y)
y = Dropout(0.5)(y)
y = add([y,skip_connection]) # adding skip connection
y = LeakyReLU()(y)
y = BatchNormalization()(y)
y = Conv2DTranspose(32, 3, activation='relu',strides=(2,2), padding='same')(y)
y = BatchNormalization()(y) #normalize y into 0 to 1
y = Dropout(0.5)(y)

output = Conv2DTranspose(3, 3, activation='sigmoid', padding='same')(x)

model4 = Model(input_img, output)
model4.summary()
Model: "model_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_3 (InputLayer)        [(None, 32, 32, 3)]       0         
                                                                 
 conv2d_10 (Conv2D)          (None, 32, 32, 32)        896       
                                                                 
 batch_normalization_11 (Bat  (None, 32, 32, 32)       128       
 chNormalization)                                                
                                                                 
 max_pooling2d_5 (MaxPooling  (None, 16, 16, 32)       0         
 2D)                                                             
                                                                 
 dropout_4 (Dropout)         (None, 16, 16, 32)        0         
                                                                 
 conv2d_11 (Conv2D)          (None, 16, 16, 64)        18496     
                                                                 
 batch_normalization_12 (Bat  (None, 16, 16, 64)       256       
 chNormalization)                                                
                                                                 
 max_pooling2d_6 (MaxPooling  (None, 8, 8, 64)         0         
 2D)                                                             
                                                                 
 dropout_5 (Dropout)         (None, 8, 8, 64)          0         
                                                                 
 conv2d_12 (Conv2D)          (None, 8, 8, 128)         73856     
                                                                 
 batch_normalization_13 (Bat  (None, 8, 8, 128)        512       
 chNormalization)                                                
                                                                 
 max_pooling2d_7 (MaxPooling  (None, 4, 4, 128)        0         
 2D)                                                             
                                                                 
 conv2d_transpose_3 (Conv2DT  (None, 8, 8, 128)        147584    
 ranspose)                                                       
                                                                 
 batch_normalization_14 (Bat  (None, 8, 8, 128)        512       
 chNormalization)                                                
                                                                 
 dropout_6 (Dropout)         (None, 8, 8, 128)         0         
                                                                 
 conv2d_transpose_4 (Conv2DT  (None, 16, 16, 64)       73792     
 ranspose)                                                       
                                                                 
 batch_normalization_15 (Bat  (None, 16, 16, 64)       256       
 chNormalization)                                                
                                                                 
 dropout_7 (Dropout)         (None, 16, 16, 64)        0         
                                                                 
 conv2d_transpose_5 (Conv2DT  (None, 32, 32, 32)       18464     
 ranspose)                                                       
                                                                 
 batch_normalization_16 (Bat  (None, 32, 32, 32)       128       
 chNormalization)                                                
                                                                 
 leaky_re_lu_1 (LeakyReLU)   (None, 32, 32, 32)        0         
                                                                 
 conv2d_13 (Conv2D)          (None, 32, 32, 3)         867       
                                                                 
 conv2d_transpose_10 (Conv2D  (None, 32, 32, 3)        84        
 Transpose)                                                      
                                                                 
=================================================================
Total params: 335,831
Trainable params: 334,935
Non-trainable params: 896
_________________________________________________________________
In [ ]:
model4.compile(optimizer=Adam(learning_rate=0.0001),metrics=['mae','accuracy'], loss="binary_crossentropy")

es = EarlyStopping(monitor='val_loss', patience=2)
history4 = model4.fit(
    noisy_train_data, 
    train_data,
    batch_size=20,
    epochs=500,
    verbose=1,
    validation_data=(noisy_test_data, test_data),
    callbacks=[es],
    shuffle=True
  )
Epoch 1/500
2500/2500 [==============================] - 52s 20ms/step - loss: 0.6026 - mae: 0.1111 - accuracy: 0.5126 - val_loss: 0.5785 - val_mae: 0.0800 - val_accuracy: 0.5862
Epoch 2/500
2500/2500 [==============================] - 52s 21ms/step - loss: 0.5781 - mae: 0.0820 - accuracy: 0.5839 - val_loss: 0.5730 - val_mae: 0.0729 - val_accuracy: 0.6190
Epoch 3/500
2500/2500 [==============================] - 51s 20ms/step - loss: 0.5752 - mae: 0.0780 - accuracy: 0.5994 - val_loss: 0.5712 - val_mae: 0.0702 - val_accuracy: 0.6277
Epoch 4/500
2500/2500 [==============================] - 51s 20ms/step - loss: 0.5740 - mae: 0.0765 - accuracy: 0.6076 - val_loss: 0.5705 - val_mae: 0.0693 - val_accuracy: 0.6339
Epoch 5/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5734 - mae: 0.0755 - accuracy: 0.6132 - val_loss: 0.5702 - val_mae: 0.0689 - val_accuracy: 0.6401
Epoch 6/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5730 - mae: 0.0749 - accuracy: 0.6167 - val_loss: 0.5700 - val_mae: 0.0687 - val_accuracy: 0.6451
Epoch 7/500
2500/2500 [==============================] - 38s 15ms/step - loss: 0.5727 - mae: 0.0745 - accuracy: 0.6194 - val_loss: 0.5698 - val_mae: 0.0685 - val_accuracy: 0.6421
Epoch 8/500
2500/2500 [==============================] - 39s 15ms/step - loss: 0.5725 - mae: 0.0742 - accuracy: 0.6214 - val_loss: 0.5695 - val_mae: 0.0678 - val_accuracy: 0.6459
Epoch 9/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5723 - mae: 0.0738 - accuracy: 0.6223 - val_loss: 0.5696 - val_mae: 0.0682 - val_accuracy: 0.6450
Epoch 10/500
2500/2500 [==============================] - 40s 16ms/step - loss: 0.5721 - mae: 0.0735 - accuracy: 0.6243 - val_loss: 0.5691 - val_mae: 0.0670 - val_accuracy: 0.6470
Epoch 11/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5720 - mae: 0.0734 - accuracy: 0.6254 - val_loss: 0.5694 - val_mae: 0.0679 - val_accuracy: 0.6466
Epoch 12/500
2500/2500 [==============================] - 39s 16ms/step - loss: 0.5719 - mae: 0.0732 - accuracy: 0.6262 - val_loss: 0.5692 - val_mae: 0.0674 - val_accuracy: 0.6490
In [ ]:
score = model4.evaluate(noisy_test_data, test_data)
print(score)
prediction_data = model4.predict(noisy_test_data)
print(f"Noisy test data and predicted output")
print(f"{display_images(noisy_test_data, prediction_data)}")
313/313 [==============================] - 4s 12ms/step - loss: 0.5692 - mae: 0.0674 - accuracy: 0.6490
[0.5691751837730408, 0.06744479387998581, 0.6490309834480286]
Noisy test data and predicted output
None
In [ ]:
index = [
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data)),
    random.randint(0, len(test_data))
]

for i in index:
    plt.subplot(1,3,1)
    plt.imshow(test_data[i])
    plt.title('original')
    plt.subplot(1,3,2)
    plt.imshow(noisy_test_data[i])
    plt.title('noisy')
    plt.subplot(1,3,3)
    plt.imshow(prediction_data[i])
    plt.title('denoised')
    plt.show()
In [ ]:
f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history4.epoch, history4.history['loss'], label = "loss") 
plt.plot(history4.epoch, history4.history['val_loss'], label = "val_loss")

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("Loss",fontsize=10)
plt.legend()
plt.show()

f = plt.figure(figsize=(5,5))
f.add_subplot()

plt.plot(history4.epoch, history4.history['mae'], label = "mae") 
plt.plot(history4.epoch, history4.history['val_mae'], label = "val_mae") 

plt.xlabel("Epochs",fontsize=10)
plt.ylabel("MAE",fontsize=10)
plt.legend()
plt.show()

We created some functions which do preprocessing task and adding noise. The goal of these functions is to rescale all images at scale. Then add noise and then display clean and noisy images side by side we create 3 separate function for these task. After all the preprocessing we created 4 models by experiemnting vaired architecture and develop autoencode models. In first model we create encoder which contains two Convolutional d layers of 32 and 64 units with BatchNormalizer and MaxPooling layers as well. We follow architecture in which there is a encoder, then a middle layer and then decoder. We used Upsampling in decoder part with Adam as optimizer with a learning_rate paramenter. UPsampling will increase the dimension and at the end of decoder will make sure that we will get the same shape as it was given as input. For the rest of the model we use Conv2dTranspose in the decoder which is the inverse of Conv2d. It do two things in layer and i.e upsampling and convolution both can be performed using this. We also tried skip connection in Model 4 as they are useful in when working with autoencoders and it helps in saving useful information which we typicallly lost when performing encoder and decoder.

-> Noise is removed but not completely, still some images classes cant classify properly and hard to interpret. These model can be improved by following the autoencoder architecture with different techniques like Upsampling and Convolution transpose and experiementing with hyperparameter. Unet autoencoder is also a good addition in autoenoder and it can help in reconstruction of images. We trited but couldn't implement perfectly.

So, in a nutshell at model 1 encoder with Conv2d, Batchnormalizer, Maxpooling and at decoder we use UpSampling2D with Conv2d, Batchnormalizer. At model 2 we use Conv2DTranspose and Dropout layers with LeakyRelu. At model 3 we use same architecture which was in model 2 but we increate number of units in both encoder & decoder. Then at model 4 added skip connection between layers and used it with Conv2d, Batchnormalizer, Maxpooling, Conv2DTranspose and Dropout layers.

All models performed well. We have tried both upsampling and Conv2DTranspose in this task. Model 4 images are much more visible as compare to other models. We used skip_connection and Conv2DTranspose layer in this technique with other layers like BatchNormalizer, Maxpooling, dropout. Using all these layers we implemented an autoencoder. Used Adam as optimizer with learning_rate parameter and analyze performance based on MAE and accuracy. At the this model give an accuracy of 72% with MAE of 0.057 and Loss of 0.56.

Object Recognition¶

object recognition model using CNN¶

In [ ]:
"""
import block
"""
import os
import torch
import torchvision
import tarfile
from torchvision.datasets.utils import download_url
from torch.utils.data import random_split
from torchvision.datasets import ImageFolder
from torchvision.transforms import ToTensor,ToPILImage
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from torch.utils.data.dataloader import DataLoader
from torchvision.utils import make_grid
import torch.nn as nn
import torch.nn.functional as F
In [ ]:
from google.colab import drive
drive.mount('/content/drive')
Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
In [ ]:
with tarfile.open('/content/drive/MyDrive/Colab Notebooks/cifar10.tgz', 'r:gz') as tar:
    tar.extractall(path='./input')
In [ ]:
"""
get the dataset and get all the labels
"""
print(os.listdir('./input/cifar10'))
classes = os.listdir('./input/cifar10/train')
print(classes)

"""
Exploring all categories
"""
for i in classes:
    class_size_path = os.path.join(f'/content/input/cifar10/train', i)
    print(f"Length of '{i.upper()}' label class = {len(os.listdir(class_size_path))}")
['test', 'labels.txt', 'train']
['ship', 'automobile', 'airplane', 'truck', 'bird', 'frog', 'deer', 'cat', 'dog', 'horse']
Length of 'SHIP' label class = 5000
Length of 'AUTOMOBILE' label class = 5000
Length of 'AIRPLANE' label class = 5000
Length of 'TRUCK' label class = 5000
Length of 'BIRD' label class = 5000
Length of 'FROG' label class = 5000
Length of 'DEER' label class = 5000
Length of 'CAT' label class = 5000
Length of 'DOG' label class = 5000
Length of 'HORSE' label class = 5000
In [ ]:
"""
save to dataset in tesor format
"""
dataPath = os.path.join(os.getcwd(), r'/content/input/cifar10/train')
dataset = ImageFolder(dataPath, transform=ToTensor())
print(dataset.classes)
['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
In [ ]:
def show_image(img,label):
    print('Label: ', dataset.classes[label],"("+str(label)+")")
    plt.imshow(img.permute(1,2,0))

show_image(dataset[222][0], dataset[222][1])
Label:  airplane (0)
In [ ]:
show_image(*dataset[1099])
Label:  airplane (0)
In [ ]:
random_seed=42
torch.manual_seed(random_seed)
val_size = 5000
train_size = len(dataset) - val_size
train_ds, val_ds = random_split(dataset,[train_size, val_size],generator=torch.manual_seed(random_seed))
len(train_ds), len(val_ds)
Out[ ]:
(45000, 5000)
In [ ]:
batch_size=128
train_dl = DataLoader(train_ds,batch_size,shuffle=True,num_workers=4,pin_memory=True )
val_dl = DataLoader(val_ds, batch_size, num_workers=4, pin_memory=True)
train_dl
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
Out[ ]:
<torch.utils.data.dataloader.DataLoader at 0x7f39747c69d0>
In [ ]:
def show_images_batch(d1):
    for images, labels in d1:
        fig, ax= plt.subplots(figsize=(16,8))
        ax.set_xticks([])
        ax.set_yticks([])
        ax.imshow(make_grid(images, nrow=16).permute(1,2,0))
        break # to stop loop otherwise 4500 images in batch size of 128 will print and is computationally expensive

show_images_batch(train_dl)
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
In [ ]:
show_images_batch(val_dl)
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
In [ ]:
def apply_kernel(image, kernel):
    ri, ci = image.shape       # image dimensions
    rk, ck = kernel.shape      # kernel dimensions
    ro, co = ri-rk+1, ci-ck+1  # output dimensions, No padding and no striding
    output = torch.zeros([ro, co])
    for i in range(ro): 
        for j in range(co):
            output[i,j] = torch.sum(image[i:i+rk,j:j+ck] * kernel) 
    return output

def accuracy(outputs,labels):
    _,preds= torch.max(outputs, dim=1)
    return torch.tensor(torch.sum(preds==labels).item()/len(preds))

class ImageClassificationBase(nn.Module):
    def training_step(self,batch):
        images,labels= batch
        out= self(images)  #Generate predictions
        loss= F.cross_entropy(out,labels) #calculate loss
        return loss
    
    def validation_step(self,batch):
        images, labels = batch
        out = self(images)  # Generate predictions
        loss= F.cross_entropy(out,labels) # calculate loss
        acc= accuracy(out,labels)
        return {'val_loss':loss.detach(), 'val_acc':acc}
    
    def validation_epoch_end(self, outputs):
        batch_losses = [x['val_loss'] for x in outputs]
        epoch_loss= torch.stack(batch_losses).mean() # Stacking losses to combine losses and calculate average
        batch_accs = [x['val_acc'] for x in outputs]
        epoch_acc = torch.stack(batch_accs).mean() # Combine accuracies
        return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}
        
    def epoch_end(self,epoch,result):
        print("Epoch [{}], train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}".format(
            epoch, result['train_loss'], result['val_loss'], result['val_acc']))

class Cifar10CnnModel(ImageClassificationBase):
    def __init__(self):
        super().__init__()
        self.network = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 64 x 16 x 16

            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 128 x 8 x 8

            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 256 x 4 x 4

            nn.Flatten(), 
            nn.Linear(256*4*4, 1024),
            nn.ReLU(),
            nn.Linear(1024, 512),
            nn.ReLU(),
            nn.Linear(512, 10))
        
    def forward(self, xb):
        return self.network(xb)

def get_default_device():
    """Pick GPU if available, else CPU"""
    if torch.cuda.is_available():
        return torch.device('cuda')
    else:
        return torch.device('cpu')
    
def to_device(data, device):
    """Move tensor(s) to chosen device"""
    if isinstance(data, (list,tuple)):
        return [to_device(x, device) for x in data]
    return data.to(device, non_blocking=True)

class DeviceDataLoader():
    """Wrap a dataloader to move data to a device"""
    def __init__(self, dl, device):
        self.dl = dl
        self.device = device
        
    def __iter__(self):
        """Yield a batch of data after moving it to device"""
        for b in self.dl: 
            yield to_device(b, self.device)

    def __len__(self):
        """Number of batches"""
        return len(self.dl)
In [ ]:
model = Cifar10CnnModel()
print(model)
Cifar10CnnModel(
  (network): Sequential(
    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU()
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (6): ReLU()
    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (8): ReLU()
    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU()
    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): ReLU()
    (14): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (15): Flatten(start_dim=1, end_dim=-1)
    (16): Linear(in_features=4096, out_features=1024, bias=True)
    (17): ReLU()
    (18): Linear(in_features=1024, out_features=512, bias=True)
    (19): ReLU()
    (20): Linear(in_features=512, out_features=10, bias=True)
  )
)
In [ ]:
device = get_default_device()
print(device)

train_dl = DeviceDataLoader(train_dl, device)
val_dl = DeviceDataLoader(val_dl, device) # load data to  device (GPU if available)
to_device(model, device) # move model to GPU if available
cuda
Out[ ]:
Cifar10CnnModel(
  (network): Sequential(
    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU()
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (6): ReLU()
    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (8): ReLU()
    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU()
    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): ReLU()
    (14): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (15): Flatten(start_dim=1, end_dim=-1)
    (16): Linear(in_features=4096, out_features=1024, bias=True)
    (17): ReLU()
    (18): Linear(in_features=1024, out_features=512, bias=True)
    (19): ReLU()
    (20): Linear(in_features=512, out_features=10, bias=True)
  )
)
In [ ]:
#In this mode, the result of every computation will have requires_grad=False, even when the inputs have requires_grad=True.
@torch.no_grad() 
def evaluate(model, val_loader):
    model.eval() # Setting model to evaluation mode, the model can adjust its behavior regarding some operations, like Dropout.
    outputs = [model.validation_step(batch) for batch in val_loader]
    return model.validation_epoch_end(outputs)
  
def fit(epochs, lr, model, train_loader, val_loader, opt_func= torch.optim.SGD):
    history=[]
    optimizer= opt_func(model.parameters(),lr) # model paramters w.r.t calculate derivative of loss
    for epoch in range(epochs):
        # Training phase
        model.train() # Setting model to training mode
        train_losses=[]
        for batch in train_loader:
            loss= model.training_step(batch)
            train_losses.append(loss)
            loss.backward() #compute  gradients
            optimizer.step()
            optimizer.zero_grad() # zero the gradients
        #Validation phase
        result= evaluate(model,val_loader)
        result['train_loss'] = torch.stack(train_losses).mean().item()
        model.epoch_end(epoch, result)
        history.append(result)
    return history

print(model)
Cifar10CnnModel(
  (network): Sequential(
    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU()
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (6): ReLU()
    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (8): ReLU()
    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU()
    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): ReLU()
    (14): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (15): Flatten(start_dim=1, end_dim=-1)
    (16): Linear(in_features=4096, out_features=1024, bias=True)
    (17): ReLU()
    (18): Linear(in_features=1024, out_features=512, bias=True)
    (19): ReLU()
    (20): Linear(in_features=512, out_features=10, bias=True)
  )
)
In [ ]:
print(evaluate(model,val_dl))
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
{'val_loss': 2.3026111125946045, 'val_acc': 0.09492187201976776}
In [ ]:
num_epochs = 15
opt_func = torch.optim.Adam
lr = 0.001
history = fit(num_epochs, lr, model, train_dl, val_dl, opt_func)
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
  cpuset_checked))
Epoch [0], train_loss: 1.7137, val_loss: 1.3119, val_acc: 0.5160
Epoch [1], train_loss: 1.1823, val_loss: 1.0236, val_acc: 0.6295
Epoch [3], train_loss: 0.7727, val_loss: 0.8302, val_acc: 0.7068
Epoch [4], train_loss: 0.6411, val_loss: 0.7651, val_acc: 0.7357
Epoch [5], train_loss: 0.5303, val_loss: 0.7066, val_acc: 0.7564
Epoch [6], train_loss: 0.4251, val_loss: 0.7120, val_acc: 0.7623
Epoch [7], train_loss: 0.3340, val_loss: 0.7165, val_acc: 0.7783
Epoch [8], train_loss: 0.2602, val_loss: 0.7612, val_acc: 0.7775
Epoch [9], train_loss: 0.2024, val_loss: 0.8355, val_acc: 0.7732
Epoch [10], train_loss: 0.1571, val_loss: 0.8716, val_acc: 0.7709
Epoch [11], train_loss: 0.1345, val_loss: 0.9591, val_acc: 0.7834
Epoch [12], train_loss: 0.1051, val_loss: 1.0662, val_acc: 0.7732
Epoch [13], train_loss: 0.0989, val_loss: 1.0044, val_acc: 0.7873
Epoch [14], train_loss: 0.0865, val_loss: 1.1470, val_acc: 0.7777
In [ ]:
def plot_accuracies(history):
    accuracies=[x['val_acc'] for x in history]
    plt.plot(accuracies,'-x')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.title('Accuracy vs No of epochs')

def plot_losses(history):
    train_losses = [x.get('train_loss') for x in history]
    val_losses = [x['val_loss'] for x in history]
    plt.plot(train_losses, '-bx')
    plt.plot(val_losses, '-rx')
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.legend(['Training', 'Validation'])
    plt.title('Loss vs. No. of epochs')
In [ ]:
plt.figure(figsize=(10,6))
plot_accuracies(history)
In [ ]:
plt.figure(figsize=(10,6))
plot_losses(history)
In [ ]:
test_dataset = ImageFolder(r'/content/input/cifar10/test', transform=ToTensor())
test_dataset
Out[ ]:
Dataset ImageFolder
    Number of datapoints: 10000
    Root location: /content/input/cifar10/test
    StandardTransform
Transform: ToTensor()
In [ ]:
def predict_image(img, model):
    xb= to_device(img.unsqueeze(0),device) # unsqueeze turns an n-dimensionsal tensor into an n+1-dimensional one. But since it is ambiguous which axis the new dimension lies across, this needs to be specified.
    # Get predictions from model
    yb = model(xb)
    # Pick index/label with highest probability
    _, preds= torch.max(yb, dim=1)
    return dataset.classes[preds[0].item()]
In [ ]:
img,label= test_dataset[8]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: airplane , Predicted: airplane
In [ ]:
img,label= test_dataset[1220]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: automobile , Predicted: automobile
In [ ]:
img,label= test_dataset[345]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: airplane , Predicted: airplane
In [ ]:
img,label= test_dataset[6153]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: frog , Predicted: frog
In [ ]:
img,label= test_dataset[456]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: airplane , Predicted: airplane
In [ ]:
img,label= test_dataset[10]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: airplane , Predicted: ship
In [ ]:
img,label= test_dataset[1432]
plt.imshow(img.permute(1,2,0))
print('Label:', dataset.classes[label], ', Predicted:', predict_image(img, model))
Label: automobile , Predicted: automobile
In [ ]:
test_loader=DeviceDataLoader(DataLoader(test_dataset,batch_size),device)
test_result = evaluate(model, test_loader)
test_result
Out[ ]:
{'val_acc': 0.7730419039726257, 'val_loss': 1.2114453315734863}